# -*- coding: utf-8 -*-
import json
import time
import scrapy
import urllib.parse

class Sohu(scrapy.Spider):
    name = 'souhu'    
    cate_num = { 'NBA广角': '442', '射门中国': '442', '足坛欧美汇': '442', '搜狐体育': '442',
                '中国篮镜头': '442', '明星权力榜': '431', '红杜鹃侃娱乐': '431', '糊说娱有料': '431', '毒药君': '431', '皮皮电影': '431', '搜狐音乐': '431',
                '时间娱乐': '431', '据说娱乐': '431', '服装搭配师main': '441', '商务范': '441', '时尚美搭展': '441', '她刊': '441', '鬼马时尚精': '441',
                '时尚搭配潮流': '441', '挑款师MK': '441', '穿搭第⑦感': '441', '墨羽青春': '441', '时尚美搭配': '441', '一街拍': '441', '超美时尚社': '441',
                '最炫前沿': '441', '辣妈时尚范': '441', '橘子搭': '441', '食客生活': '440', '舞之灵私房菜': '440', '爱家庭美食': '440', '香儿厨房': '440',
                '鲜衣美食君': '440', '券商中国': '445', '和讯网': '445', '搜狐财经快讯': '445', '朱邦凌': '445', '第一白银网': '445', '搜狐科技视界': '433',
                '果壳网': '433', '科技说说': '433', '科技小李': '433', '创业家精选': '433',
                }

    def start_requests(self):
        user_dict = {'NBA广角': 'cHBhZzU4NjJkMmIzNGIzZUBzb2h1LmNvbQ==',
                     '射门中国': 'cHBhZzkyMDM3OWNhMjc0MEBzb2h1LmNvbQ==', '足坛欧美汇': 'cHBhZzQ0NzIzNDQwNjhhM0Bzb2h1LmNvbQ==',
                     '搜狐体育': 'c29odXptdHlwbHRodEBzb2h1LmNvbQ==', '中国篮镜头': 'Y2JhY2hpbmFAc29odS5jb20=',
                     '明星权力榜': 'NTA1Nzg4NTE0M0BzaW5hLnNvaHUuY29t', '红杜鹃侃娱乐': 'OTc5OTY0MjEyNjM1OTQyOTEyQHNvaHUuY29t',
                     '糊说娱有料': 'cHBhZzA5MTQ2ZjFlODNmZEBzb2h1LmNvbQ==', '毒药君': 'NTI2MzMwOTc3NUBzaW5hLnNvaHUuY29t',
                     '皮皮电影': 'MTI2MDQzMDY2NEBzaW5hLnNvaHUuY29t', '搜狐音乐': 'c29odXptdGVxdnRyakBzb2h1LmNvbQ==',
                     '时间娱乐': 'MTAxNzYyOTQyNTgxNjE1ODIwOEBzb2h1LmNvbQ==', '据说娱乐': 'cHBhZzU2NDY4NGUwMzUxZEBzb2h1LmNvbQ==',
                     '服装搭配师main': 'MTE5ODk3NjY2MEBzaW5hLnNvaHUuY29t',
                     '商务范': 'NkY3MDFDRkU4N0Q4NUYyMzQzOThEMTZDREYxNjMwRjFAcXEuc29odS5jb20=',
                     '时尚美搭展': 'YmFnYWltdWRpQHNvaHUuY29t=', '她刊': 'Y29vbGRiMkAxMjYuY29t=',
                     '鬼马时尚精': 'cHBhZzg2MDc2MTQyOWQ5M0Bzb2h1LmNvbQ===', '观察者网': 'MTg4NzM0NDM0MUBzaW5hLnNvaHUuY29t',
                     '时尚搭配潮流': 'OTgwNTcyNjY1REQ5MEE3MkU0MTYyRDFERTM3NURCRDdAcXEuc29odS5jb20=',
                     '挑款师MK': 'cHBhZzIxMzQzNTRhNDU5MUBzb2h1LmNvbQ==', '穿搭第⑦感': 'Y2ljaGFvMzk4NDU1NUBzb2h1LmNvbQ==',
                     '墨羽青春': 'MTUyNTAwNTM2OTNAc29odS5jb20=',
                     '时尚美搭配': 'RTJFOUI0M0Y4RkY2QkFGMTk1NDc5MDg3OEY0RUVEQ0NAcXEuc29odS5jb20=',
                     '一街拍': 'cHBhZzU5MjhhNDM0MTRhOEBzb2h1LmNvbQ==', '超美时尚社': 'c29odXptdDdxbmtlYzhAc29odS5jb20=',
                     '最炫前沿': 'NkVFODA5RTEwNzlGODg2NzdFRkVGNzc3NkFFRkU5OThAcXEuc29odS5jb20=',
                     '辣妈时尚范': 'cHBhZzMwMzc5ZDBhNjBmZUBzb2h1LmNvbQ==',
                     '橘子搭': 'RDRDMkFFMzg1Mzg4NEVFQThFRDMyOTY5ODExRDREM0FAcXEuc29odS5jb20=',
                     '食客生活': 'cHBhZzA1NjNlNTVmMmU5MEBzb2h1LmNvbQ===', '舞之灵私房菜': 'eGh5X21tQHNvaHUuY29t=',
                     '爱家庭美食': 'eWluaHVhNzdAMTI2LmNvbQ===',
                     '香儿厨房': 'MTY5NkNFMTlBQUM2MDdCMTFDOEY3MzQwQjhDQUQ3MTRAcXEuc29odS5jb20==',
                     '鲜衣美食君': 'c29odXptdDZxb29nam1Ac29odS5jb20==', '券商中国': 'c29odXptdHB2YnkycnhAc29odS5jb20==',
                     '和讯网': 'cHBhZzUxNjJjYzY2ZmFiNkBzb2h1LmNvbQ===',
                     '搜狐财经快讯': 'MTAwMDk0OTc1NTYzNzA4MDA2NEBzb2h1LmNvbQ===',
                     '朱邦凌': 'b1NlSFRzMEE4dDk3MW1RMnhLdUZrXzNpQzdrUUB3ZWNoYXQuc29odS5jb20==',
                     '第一白银网': 'cHBhZzE2ODM2MDQwYWRkM0Bzb2h1LmNvbQ===', '搜狐科技视界': 'a2VqaXF1YW5qaWFud2VuQHNvaHUuY29t=',
                     '果壳网': 'c29odXptdGhnbjZ3NEBzb2h1LmNvbQ===', '科技说说': 'Y2ljbzIwMDRAMTYzLmNvbQ===',
                     '科技小李': 'OTcxRTgyNjVGNjFFRDYzMTNDNkEzRTMwQzFGQjQ4QzFAcXEuc29odS5jb20==',
                     '创业家精选': 'MTAwMzU0ODMzMTk0OTI2ODk5MkBzb2h1LmNvbQ==='
                     }
        base_url1 = 'http://mp.sohu.com/apiV2/profile/newsListAjax?xpt={}&pageNumber=1&pageSize=10'
        for i in user_dict:
            time.sleep(3)
            item = {}
            item['source'] = i
            url = base_url1.format(user_dict[i])
            yield scrapy.Request(url, callback=self.parse, meta={"item": item})

    def parse(self, response):
        item = response.meta["item"]
        cate = item['source']
        con = response.text.replace('\\', '')[1:-1]
        content = json.loads(urllib.parse.unquote(con))
        for i in content['data']:
            item = {}
            item['title'] = i['title']
            item['url'] = "http:" + i['url']
            item['browse_count'] = i['newsPv']  # 阅读量
            item['comment_count'] = i['commentsCnt']  # 评论数
            item['publish_time'] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(str(i['postTime'])[:-3])))
            item['spider_time'] = time.strftime("%Y-%m-%d %X", time.localtime())
            item['author'] = cate
            item['site_id'] = "113"  # 站点分类
            item['site_classify_id'] = self.cate_num[cate]
            # print(item)
            yield item
